{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zsh:1: command not found: nvidia-smi\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaTokenizer, RobertaTokenizerFast\n",
    "from transformers import RobertaForSequenceClassification\n",
    "from transformers import RobertaConfig, RobertaModel, RobertaForMaskedLM\n",
    "from transformers import LineByLineTextDataset, DataCollatorForLanguageModeling\n",
    "from transformers import Trainer, TrainingArguments\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "maskedlm_model = RobertaForMaskedLM.from_pretrained(\"roberta-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RobertaTokenizerFast.from_pretrained(\"roberta-large\", do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['c', 'ov', 'id']\n",
      "['cor', 'on', 'av', 'irus']\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.tokenize('covid'))\n",
    "print(tokenizer.tokenize('coronavirus'))\n",
    "print(tokenizer.tokenize('cdc'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50265\n",
      "50267\n"
     ]
    }
   ],
   "source": [
    "print (len(tokenizer)) \n",
    "tokenizer.add_tokens([\"covid\"]) \n",
    "tokenizer.add_tokens([\"coronavirus\"])\n",
    "tokenizer.add_tokens([\"cdc\"])\n",
    "print (len(tokenizer)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['covid']\n",
      "['coronavirus']\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.tokenize('covid'))\n",
    "print(tokenizer.tokenize('coronavirus'))\n",
    "print(tokenizer.tokenize('cdc'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['covid', 'Ġand', 'Ġ', 'coronavirus', 'Ġare', 'Ġbad']\n"
     ]
    }
   ],
   "source": [
    "test_sentence = \"covid and coronavirus are bad, cdc is good authority truth on health\"\n",
    "print(tokenizer.tokenize(test_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGzCAYAAACPa3XZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvBUlEQVR4nO3de1yU1d7//zcHGUgEQhFE8ZjlKbUwldJMpciwNOlkWHjYmVuqnZal9y7N9r7Drd5ptTG770w7mYWZh9plecjK0NS2bQ/JQ01TUzAlQE0BZX3/6Mf8nEBldGbh0Ov5eMxj71nXmuv6XIvJeXPNWlx+xhgjAAAAS/yruwAAAPDHQvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4ADxk8ODBatq06R/u2L/3zDPPyM/Pr0p958yZIz8/P+3evdvt41R2zn5+fnrmmWfc3teFcOd8AfyG8AGfU/6BVf4IDAxUw4YNNXjwYP3000/VXV6N0LRpU/Xt29dj+3vuuee0cOFCj+3Pl5SWlqpevXrq1q3bGfsYYxQXF6err77ao8fev3+/nnnmGW3cuNGj+wUuFOEDPuvZZ5/Vm2++qZkzZ6pPnz5666231KNHD504caK6S/tDe+qpp3T8+HGXtjOFj/vuu0/Hjx9XkyZNPHLs48eP66mnnvLIvjylVq1auvPOO/X111/rxx9/rLTPF198oX379mnQoEEePfb+/fs1ceJEwgcuOoQP+Kw+ffpo0KBB+tOf/qRXX31Vjz/+uHbu3KnFixdXd2l/aIGBgQoODq5S34CAAAUHB3vsa4vg4GAFBgZ6ZF+elJqaKmOM3nnnnUq3z507V/7+/rrnnnssV3Z+jh07Vt0lwMcRPlBjdO/eXZK0c+dOZ1tJSYnGjx+v+Ph4hYeHq3bt2urevbtWrlzp8trdu3fLz89PU6dO1f/+7/+qRYsWcjgcuuaaa7Ru3boKx1q4cKHatWun4OBgtWvXTh988EGlNR07dkyPPfaY4uLi5HA4dMUVV2jq1Kn6/c2k/fz89NBDDykrK0tt2rRRSEiIEhIStGnTJknSK6+8ossuu0zBwcG64YYbzjpHwhijpk2bql+/fhW2nThxQuHh4XrwwQfP+PrKuDM+v58D4efnp2PHjun11193flU2ePBgSZXP+Vi0aJGSk5MVGxsrh8OhFi1a6G9/+5tOnTp1zjpPn/NRXvOZHqdbu3atbr75ZoWHh+uSSy5Rjx49tHr16gr7/+qrr3TNNdcoODhYLVq00CuvvFKl8bvuuuvUtGlTzZ07t8K20tJSzZ8/Xz179lRsbKwkadu2bbrjjjsUGRmp4OBgderUqdJQXVBQoFGjRqlp06ZyOBxq1KiR7r//fh06dEiff/65rrnmGknSkCFDnOc9Z84c5+uzsrIUHx+vkJAQ1atXT4MGDarw1eXgwYMVGhqqnTt36pZbblGdOnWUmppapfMGzuTi+xUBOE/lH2CXXnqps62oqEivvvqqBg4cqAceeEBHjhzRrFmzlJSUpG+++UYdO3Z02cfcuXN15MgRPfjgg/Lz89PkyZM1YMAA/fDDD6pVq5Yk6dNPP1VKSoratGmjjIwMHT58WEOGDFGjRo1c9mWM0W233aaVK1dq2LBh6tixo5YuXaoxY8bop59+0rRp01z6f/nll1q8eLHS09MlSRkZGerbt6+eeOIJzZgxQyNHjtQvv/yiyZMna+jQoVqxYkWl4+Dn56dBgwZp8uTJys/PV2RkpHPbkiVLVFRUdN6X96syPr/35ptv6k9/+pM6d+6s4cOHS5JatGhxxmPMmTNHoaGhGj16tEJDQ7VixQqNHz9eRUVFmjJlSpVrjYqK0ptvvunSVlpaqlGjRikoKMjZtmLFCvXp00fx8fGaMGGC/P39NXv2bPXq1UtffvmlOnfuLEnatGmTbrrpJkVFRemZZ57RyZMnNWHCBEVHR5+zFj8/P91777167rnntGXLFrVt29a57ZNPPlF+fr7zA33Lli267rrr1LBhQ40dO1a1a9fWe++9p/79++v999/X7bffLkk6evSounfvru+//15Dhw7V1VdfrUOHDmnx4sXat2+fWrdurWeffVbjx4/X8OHDneH82muvdY7zkCFDdM011ygjI0N5eXl64YUXtHr1av373/9WRESEs8aTJ08qKSlJ3bp109SpU3XJJZdU+ecAVMoAPmb27NlGklm2bJn5+eefzd69e838+fNNVFSUcTgcZu/evc6+J0+eNMXFxS6v/+WXX0x0dLQZOnSos23Xrl1Gkqlbt67Jz893ti9atMhIMkuWLHG2dezY0TRo0MAUFBQ42z799FMjyTRp0sTZtnDhQiPJ/P3vf3c5/h133GH8/PzMjh07nG2SjMPhMLt27XK2vfLKK0aSiYmJMUVFRc72cePGGUkufdPS0lyOnZOTYySZl19+2eXYt912m2natKkpKyv7/bC6aNKkiUlOTj6v8ZkwYYL5/T8ttWvXNmlpaRWOU/6zPP1cfv311wr9HnzwQXPJJZeYEydOnPGcjfltHCdMmHDG8xo5cqQJCAgwK1asMMYYU1ZWZlq2bGmSkpJcxuTXX381zZo1MzfeeKOzrX///iY4ONj8+OOPzratW7eagICACudbmS1bthhJZty4cS7t99xzjwkODjaFhYXGGGN69+5trrzySpdzLSsrM9dee61p2bKls238+PFGklmwYEGFY5Wfy7p164wkM3v2bJftJSUlpn79+qZdu3bm+PHjzvYPP/zQSDLjx493tqWlpRlJZuzYsec8R6Cq+NoFPisxMVFRUVGKi4vTHXfcodq1a2vx4sUuVyACAgKcv+WWlZUpPz9fJ0+eVKdOnfTtt99W2Ofdd9/tcuWk/LfFH374QZJ04MABbdy4UWlpaQoPD3f2u/HGG9WmTRuXff3rX/9SQECAHnnkEZf2xx57TMYYffzxxy7tvXv3dlk62qVLF0lSSkqK6tSpU6G9vKbKXH755erSpYvefvttZ1t+fr4+/vhjpaamnvcci3ONjyeEhIQ4//+RI0d06NAhde/eXb/++qu2bdt23vt94403NGPGDE2ePFk9e/aUJG3cuFHbt2/Xvffeq8OHD+vQoUM6dOiQjh07pt69e+uLL75QWVmZTp06paVLl6p///5q3Lixc5+tW7dWUlJSlY7fpk0bXXXVVZo3b56z7dixY1q8eLH69u2rsLAw5efna8WKFbrrrruc537o0CEdPnxYSUlJ2r59u/Nrkffff18dOnRwXgk53bl+vuvXr9fBgwc1cuRIl/k5ycnJatWqlT766KMKr/nzn/9cpfMEqoLwAZ+VmZmpzz77TPPnz9ctt9yiQ4cOyeFwVOj3+uuvq3379goODlbdunUVFRWljz76SIWFhRX6nv7BIv3/X+H88ssvkuRcrdCyZcsKr73iiitcnv/444+KjY11CQ7Sbx9Yp+/rTMcuDzdxcXGVtpfXdCb333+/Vq9e7TxOVlaWSktLdd999531dWdzrvHxhC1btuj2229XeHi4wsLCFBUV5fyaqLKfWVVs3LhRI0aM0MCBAzV69Ghn+/bt2yVJaWlpioqKcnm8+uqrKi4uVmFhoX7++WcdP368Sj/3s0lNTdWuXbv09ddfS/pt7tCvv/7q/Mplx44dMsbo6aefrlDPhAkTJEkHDx6U9Nvcpnbt2p3XeJS/JyqrvVWrVhXem4GBgRW+VgQuBHM+4LM6d+6sTp06SZL69++vbt266d5771VOTo5CQ0MlSW+99ZYGDx6s/v37a8yYMapfv74CAgKUkZHhMjG1XEBAQKXHMr+bIOoNZzr2+dZ0zz33aNSoUXr77bf1X//1X3rrrbfUqVMntz4sPVVLVRUUFKhHjx4KCwvTs88+qxYtWig4OFjffvutnnzySZWVlbm9z19++UUpKSm6/PLL9eqrr7psK9/flClTKsz/KRcaGqri4mK3j1uZgQMH6oknntDcuXN17bXXau7cubr00kt1yy23uNTz+OOPn/GKymWXXeaRWtzhcDjk78/vqvAcwgdqhPJA0bNnT/3zn//U2LFjJUnz589X8+bNtWDBApdL0eW/Rbqr/O9RlP/GfLqcnJwKfZctW6YjR464XP0o/+rAU3/b4kwiIyOVnJyst99+W6mpqVq9erWmT5/u1WOeSVW/5vn88891+PBhLViwQNdff72zfdeuXed13LKyMqWmpqqgoEDLli2rMFGyfOJrWFiYEhMTz7ifqKgohYSEVOnnfjaxsbHq2bOnsrKy9PTTT+uzzz7T4MGDnV8NNm/eXNJvfxvkbPWU17558+az9jnTuJe/93JyctSrVy+XbTk5OV5/bwJEWdQYN9xwgzp37qzp06c7/9BY+W/qp/9mvnbtWmVnZ5/XMRo0aKCOHTvq9ddfd/kK4LPPPtPWrVtd+t5yyy06deqU/vnPf7q0T5s2TX5+furTp8951eCO++67T1u3btWYMWMUEBBQbX9Honbt2iooKDhnv8p+XiUlJZoxY8Z5HXfixIlaunSp3nnnHTVr1qzC9vj4eLVo0UJTp07V0aNHK2z/+eefnXUlJSVp4cKF2rNnj3P7999/r6VLl7pVU2pqqg4ePKgHH3xQpaWlLstW69evrxtuuEGvvPKKDhw4cMZ6pN/mAn333XeVLvMuH7/atWtLUoWx79Spk+rXr6+ZM2e6XNX5+OOP9f333ys5OdmtcwLcxZUP1ChjxozRnXfeqTlz5mjEiBHq27evFixYoNtvv13JycnatWuXZs6cqTZt2lT6YVMVGRkZSk5OVrdu3TR06FDl5+frpZdeUtu2bV32eeutt6pnz57661//qt27d6tDhw769NNPtWjRIj366KNnXW7qKcnJyapbt66ysrLUp08f1a9f3+vHrEx8fLyWLVum559/XrGxsWrWrJlz4uzprr32Wl166aVKS0vTI488Ij8/P7355pvn9bXOpk2b9Le//U3XX3+9Dh48qLfeestl+6BBg+Tv769XX31Vffr0Udu2bTVkyBA1bNhQP/30k1auXKmwsDAtWbJE0m9B5pNPPlH37t01cuRInTx50vlz/89//lPlulJSUjRy5EgtWrRIcXFxLld4pN/mMnXr1k1XXnmlHnjgATVv3lx5eXnKzs7Wvn379N1330n67b0+f/583XnnnRo6dKji4+OVn5+vxYsXa+bMmerQoYNatGihiIgIzZw5U3Xq1FHt2rXVpUsXNWvWTP/4xz80ZMgQ9ejRQwMHDnQutW3atKlGjRrl9ngDbqm2dTbAeSpfnrlu3boK206dOmVatGhhWrRoYU6ePGnKysrMc889Z5o0aWIcDoe56qqrzIcfflhhmWb5UtIpU6ZU2KcqWb75/vvvm9atWxuHw2HatGljFixYUOnSzyNHjphRo0aZ2NhYU6tWLdOyZUszZcqUCktdJZn09HSXtjPVtHLlSiPJZGVlOdsqO3a5kSNHGklm7ty5lW6vzJmW2lZlfCpbartt2zZz/fXXm5CQECPJuey2sqW2q1evNl27djUhISEmNjbWPPHEE2bp0qVGklm5cuVZz/n0WsrH6UyP0/373/82AwYMMHXr1jUOh8M0adLE3HXXXWb58uUu/VatWmXi4+NNUFCQad68uZk5c2al53sud955p5FknnjiiUq379y509x///0mJibG1KpVyzRs2ND07dvXzJ8/36Xf4cOHzUMPPWQaNmxogoKCTKNGjUxaWpo5dOiQs8+iRYtMmzZtTGBgYIVlt++++6656qqrjMPhMJGRkSY1NdXs27fP5RhpaWmmdu3abp0fcC5+xliYSQeg2owaNUqzZs1Sbm4ufxwKwEWBOR9ADXbixAm99dZbSklJIXgAuGgw5wOogQ4ePKhly5Zp/vz5Onz4sP7yl79Ud0kA4ET4AGqgrVu3KjU1VfXr19eLL754xr9hAQDVgTkfAADAKuZ8AAAAqwgfAADAqotuzkdZWZn279+vOnXqnPedNwEAgF3GGB05ckSxsbHnvBfQRRc+9u/fX+EungAAwDfs3bv3nHdBvujCR/kNuPbu3auwsLBqrgYAAFRFUVGR4uLiXG6keSYXXfgo/6olLCyM8AEAgI+pypQJJpwCAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMCqwOouAAC8penYj87ZZ/ekZAuVADgdVz4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVgdVdAIA/lqZjPzpnn92Tki1UAqC6cOUDAABY5Vb4eOaZZ+Tn5+fyaNWqlXP7iRMnlJ6errp16yo0NFQpKSnKy8vzeNEAAMB3uX3lo23btjpw4IDz8dVXXzm3jRo1SkuWLFFWVpZWrVql/fv3a8CAAR4tGAAA+Da353wEBgYqJiamQnthYaFmzZqluXPnqlevXpKk2bNnq3Xr1lqzZo26du164dUCAACf5/aVj+3btys2NlbNmzdXamqq9uzZI0nasGGDSktLlZiY6OzbqlUrNW7cWNnZ2WfcX3FxsYqKilweAACg5nLrykeXLl00Z84cXXHFFTpw4IAmTpyo7t27a/PmzcrNzVVQUJAiIiJcXhMdHa3c3Nwz7jMjI0MTJ048r+IB1EysiAFqNrfCR58+fZz/v3379urSpYuaNGmi9957TyEhIedVwLhx4zR69Gjn86KiIsXFxZ3XvgAAwMXvgpbaRkRE6PLLL9eOHTsUExOjkpISFRQUuPTJy8urdI5IOYfDobCwMJcHAACouS4ofBw9elQ7d+5UgwYNFB8fr1q1amn58uXO7Tk5OdqzZ48SEhIuuFAAAFAzuPW1y+OPP65bb71VTZo00f79+zVhwgQFBARo4MCBCg8P17BhwzR69GhFRkYqLCxMDz/8sBISEljpAgAAnNwKH/v27dPAgQN1+PBhRUVFqVu3blqzZo2ioqIkSdOmTZO/v79SUlJUXFyspKQkzZgxwyuFAwAA3+RW+Jg3b95ZtwcHByszM1OZmZkXVBQAAKi5uLcLAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKxya6ktAMC7qnJTPYkb68G3ceUDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFjFahcAOIeqrEBh9QlQdVz5AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVLLUF8IdW1Ru5AfAcrnwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwKrA6i4AAOC+pmM/Omef3ZOSLVQCuI8rHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKu4sRwAn1SVG6sBuDhx5QMAAFh1QeFj0qRJ8vPz06OPPupsO3HihNLT01W3bl2FhoYqJSVFeXl5F1onAACoIc47fKxbt06vvPKK2rdv79I+atQoLVmyRFlZWVq1apX279+vAQMGXHChAACgZjiv8HH06FGlpqbq//7v/3TppZc62wsLCzVr1iw9//zz6tWrl+Lj4zV79mx9/fXXWrNmjceKBgAAvuu8wkd6erqSk5OVmJjo0r5hwwaVlpa6tLdq1UqNGzdWdnZ2pfsqLi5WUVGRywMAANRcbq92mTdvnr799lutW7euwrbc3FwFBQUpIiLCpT06Olq5ubmV7i8jI0MTJ050twwAAOCj3LrysXfvXv3lL3/R22+/reDgYI8UMG7cOBUWFjofe/fu9ch+AQDAxcmt8LFhwwYdPHhQV199tQIDAxUYGKhVq1bpxRdfVGBgoKKjo1VSUqKCggKX1+Xl5SkmJqbSfTocDoWFhbk8AABAzeXW1y69e/fWpk2bXNqGDBmiVq1a6cknn1RcXJxq1aql5cuXKyUlRZKUk5OjPXv2KCEhwXNVAwAAn+VW+KhTp47atWvn0la7dm3VrVvX2T5s2DCNHj1akZGRCgsL08MPP6yEhAR17drVc1UDAACf5fE/rz5t2jT5+/srJSVFxcXFSkpK0owZMzx9GAAA4KMuOHx8/vnnLs+Dg4OVmZmpzMzMC901AACogbi3CwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACs8vjf+QDwx9V07EfVXcJFjfEBfsOVDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFXcWA6o4apyM7Pdk5ItVAIAv+HKBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwitUuAPAHxmooVAeufAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAqsDqLgBA5ZqO/eicfXZPSrZQyW+qUg8AVAVXPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFUttAbCMtobi54qLFVc+AACAVW6Fj5dfflnt27dXWFiYwsLClJCQoI8//ti5/cSJE0pPT1fdunUVGhqqlJQU5eXlebxoAADgu9wKH40aNdKkSZO0YcMGrV+/Xr169VK/fv20ZcsWSdKoUaO0ZMkSZWVladWqVdq/f78GDBjglcIBAIBvcmvOx6233ury/L//+7/18ssva82aNWrUqJFmzZqluXPnqlevXpKk2bNnq3Xr1lqzZo26du3quaoBAIDPOu85H6dOndK8efN07NgxJSQkaMOGDSotLVViYqKzT6tWrdS4cWNlZ2efcT/FxcUqKipyeQAAgJrL7fCxadMmhYaGyuFwaMSIEfrggw/Upk0b5ebmKigoSBERES79o6OjlZube8b9ZWRkKDw83PmIi4tz+yQAAIDvcDt8XHHFFdq4caPWrl2rP//5z0pLS9PWrVvPu4Bx48apsLDQ+di7d+957wsAAFz83P47H0FBQbrsssskSfHx8Vq3bp1eeOEF3X333SopKVFBQYHL1Y+8vDzFxMSccX8Oh0MOh8P9ygEAgE+64L/zUVZWpuLiYsXHx6tWrVpavny5c1tOTo727NmjhISECz0MAACoIdy68jFu3Dj16dNHjRs31pEjRzR37lx9/vnnWrp0qcLDwzVs2DCNHj1akZGRCgsL08MPP6yEhARWugAAACe3wsfBgwd1//3368CBAwoPD1f79u21dOlS3XjjjZKkadOmyd/fXykpKSouLlZSUpJmzJjhlcIBAIBvcit8zJo166zbg4ODlZmZqczMzAsqCgAA1FzcWA4APICbuAFVx43lAACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWMW9XYBqwH1A4Euq8n7dPSnZQiWoKbjyAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrWGoL+DCW7KKmYVnvHwNXPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVboWPjIwMXXPNNapTp47q16+v/v37Kycnx6XPiRMnlJ6errp16yo0NFQpKSnKy8vzaNEAAMB3uRU+Vq1apfT0dK1Zs0afffaZSktLddNNN+nYsWPOPqNGjdKSJUuUlZWlVatWaf/+/RowYIDHCwcAAL4p0J3On3zyicvzOXPmqH79+tqwYYOuv/56FRYWatasWZo7d6569eolSZo9e7Zat26tNWvWqGvXrp6rHAAA+KQLmvNRWFgoSYqMjJQkbdiwQaWlpUpMTHT2adWqlRo3bqzs7OxK91FcXKyioiKXBwAAqLncuvJxurKyMj366KO67rrr1K5dO0lSbm6ugoKCFBER4dI3Ojpaubm5le4nIyNDEydOPN8ygItO07EfVXcJAHBRO+8rH+np6dq8ebPmzZt3QQWMGzdOhYWFzsfevXsvaH8AAODidl5XPh566CF9+OGH+uKLL9SoUSNne0xMjEpKSlRQUOBy9SMvL08xMTGV7svhcMjhcJxPGQAAwAe5deXDGKOHHnpIH3zwgVasWKFmzZq5bI+Pj1etWrW0fPlyZ1tOTo727NmjhIQEz1QMAAB8mltXPtLT0zV37lwtWrRIderUcc7jCA8PV0hIiMLDwzVs2DCNHj1akZGRCgsL08MPP6yEhARWugAAAEluho+XX35ZknTDDTe4tM+ePVuDBw+WJE2bNk3+/v5KSUlRcXGxkpKSNGPGDI8UCwAAfJ9b4cMYc84+wcHByszMVGZm5nkXBQAAaq7zXmoLAEC5qiwx3z0p2UIlv7nY6oErbiwHAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKxitQv+EDx1szdmxwPAhePKBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsCqzuAoCzaTr2o3P22T0p2UIlv6lKPQCAs+PKBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCpuLAcA8Cnc4NH3ceUDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFjFahf4PGa+A4Bv4coHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKxiqS0AwIqLbVl8VerZPSnZQiV/PFz5AAAAVrkdPr744gvdeuutio2NlZ+fnxYuXOiy3Rij8ePHq0GDBgoJCVFiYqK2b9/uqXoBAICPczt8HDt2TB06dFBmZmal2ydPnqwXX3xRM2fO1Nq1a1W7dm0lJSXpxIkTF1wsAADwfW7P+ejTp4/69OlT6TZjjKZPn66nnnpK/fr1kyS98cYbio6O1sKFC3XPPfdcWLUAAMDneXTOx65du5Sbm6vExERnW3h4uLp06aLs7OxKX1NcXKyioiKXBwAAqLk8utolNzdXkhQdHe3SHh0d7dz2exkZGZo4caIny4CPuNhmvgMA7Kj21S7jxo1TYWGh87F3797qLgkAAHiRR8NHTEyMJCkvL8+lPS8vz7nt9xwOh8LCwlweAACg5vJo+GjWrJliYmK0fPlyZ1tRUZHWrl2rhIQETx4KAAD4KLfnfBw9elQ7duxwPt+1a5c2btyoyMhINW7cWI8++qj+/ve/q2XLlmrWrJmefvppxcbGqn///p6sGwAA+Ci3w8f69evVs2dP5/PRo0dLktLS0jRnzhw98cQTOnbsmIYPH66CggJ169ZNn3zyiYKDgz1XNQAA8Fl+xhhT3UWcrqioSOHh4SosLGT+Rw3HahcAFzvu7VJ17nx+c2M5AAAuADeoc1+1L7UFAAB/LIQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWMW9XeAV3DQOAHAmXPkAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFUstYXbWEYLAO6pyr+buyclW6jk4sCVDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFatd/kCYbQ0A7mF1n3dw5QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAVN5bzIl+8kRs3UQKAi5cvfq5UhisfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqVrsAAFCD+MKKGK58AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMCqP9xSW08tQfLUDdgutnoAANXjj/TvOFc+AACAVV4LH5mZmWratKmCg4PVpUsXffPNN946FAAA8CFeCR/vvvuuRo8erQkTJujbb79Vhw4dlJSUpIMHD3rjcAAAwId4JXw8//zzeuCBBzRkyBC1adNGM2fO1CWXXKLXXnvNG4cDAAA+xOMTTktKSrRhwwaNGzfO2ebv76/ExERlZ2dX6F9cXKzi4mLn88LCQklSUVGRp0uTJJUV/3rOPlU5dlX24ykXWz0AAN/mjc/Y8n0aY87Z1+Ph49ChQzp16pSio6Nd2qOjo7Vt27YK/TMyMjRx4sQK7XFxcZ4urcrCp1fboSt1sdUDAPBt3vxcOXLkiMLDw8/ap9qX2o4bN06jR492Pi8rK1N+fr7q1q0rPz+/aqzMfUVFRYqLi9PevXsVFhZW3eVc9BivqmOsqo6xqjrGquoYq3MzxujIkSOKjY09Z1+Ph4969eopICBAeXl5Lu15eXmKiYmp0N/hcMjhcLi0RUREeLosq8LCwnhzuoHxqjrGquoYq6pjrKqOsTq7c13xKOfxCadBQUGKj4/X8uXLnW1lZWVavny5EhISPH04AADgY7zytcvo0aOVlpamTp06qXPnzpo+fbqOHTumIUOGeONwAADAh3glfNx99936+eefNX78eOXm5qpjx4765JNPKkxCrWkcDocmTJhQ4WskVI7xqjrGquoYq6pjrKqOsfIsP1OVNTEAAAAewr1dAACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA835efnKzU1VWFhYYqIiNCwYcN09OjRs77mxIkTSk9PV926dRUaGqqUlJQKfwFWkubMmaP27dsrODhY9evXV3p6urdOwwpvjpUkHT58WI0aNZKfn58KCgq8cAb2eGOsvvvuOw0cOFBxcXEKCQlR69at9cILL3j7VDwuMzNTTZs2VXBwsLp06aJvvvnmrP2zsrLUqlUrBQcH68orr9S//vUvl+3GGI0fP14NGjRQSEiIEhMTtX37dm+eglWeHK/S0lI9+eSTuvLKK1W7dm3Fxsbq/vvv1/79+719GlZ4+r11uhEjRsjPz0/Tp0/3cNU1hIFbbr75ZtOhQwezZs0a8+WXX5rLLrvMDBw48KyvGTFihImLizPLly8369evN127djXXXnutS5//+Z//MbGxsebtt982O3bsMN99951ZtGiRN0/F67w1VuX69etn+vTpYySZX375xQtnYI83xmrWrFnmkUceMZ9//rnZuXOnefPNN01ISIh56aWXvH06HjNv3jwTFBRkXnvtNbNlyxbzwAMPmIiICJOXl1dp/9WrV5uAgAAzefJks3XrVvPUU0+ZWrVqmU2bNjn7TJo0yYSHh5uFCxea7777ztx2222mWbNm5vjx47ZOy2s8PV4FBQUmMTHRvPvuu2bbtm0mOzvbdO7c2cTHx9s8La/wxnur3IIFC0yHDh1MbGysmTZtmpfPxDcRPtywdetWI8msW7fO2fbxxx8bPz8/89NPP1X6moKCAlOrVi2TlZXlbPv++++NJJOdnW2MMSY/P9+EhISYZcuWefcELPLWWJWbMWOG6dGjh1m+fLnPhw9vj9XpRo4caXr27Om54r2sc+fOJj093fn81KlTJjY21mRkZFTa/6677jLJyckubV26dDEPPvigMcaYsrIyExMTY6ZMmeLcXlBQYBwOh3nnnXe8cAZ2eXq8KvPNN98YSebHH3/0TNHVxFtjtW/fPtOwYUOzefNm06RJE8LHGfC1ixuys7MVERGhTp06OdsSExPl7++vtWvXVvqaDRs2qLS0VImJic62Vq1aqXHjxsrOzpYkffbZZyorK9NPP/2k1q1bq1GjRrrrrru0d+9e756QF3lrrCRp69atevbZZ/XGG2/I39/338LeHKvfKywsVGRkpOeK96KSkhJt2LDB5Rz9/f2VmJh4xnPMzs526S9JSUlJzv67du1Sbm6uS5/w8HB16dLlrOPmC7wxXpUpLCyUn5+fT98A1FtjVVZWpvvuu09jxoxR27ZtvVN8DeH7/3JblJubq/r167u0BQYGKjIyUrm5uWd8TVBQUIX/UKOjo52v+eGHH1RWVqbnnntO06dP1/z585Wfn68bb7xRJSUlXjkXb/PWWBUXF2vgwIGaMmWKGjdu7JXabfPWWP3e119/rXfffVfDhw/3SN3edujQIZ06darCbRnOdo65ubln7V/+v+7s01d4Y7x+78SJE3ryySc1cOBAn76zq7fG6h//+IcCAwP1yCOPeL7oGobwIWns2LHy8/M762Pbtm1eO35ZWZlKS0v14osvKikpSV27dtU777yj7du3a+XKlV477vmo7rEaN26cWrdurUGDBnntGJ5S3WN1us2bN6tfv36aMGGCbrrpJivHRM1SWlqqu+66S8YYvfzyy9VdzkVnw4YNeuGFFzRnzhz5+flVdzkXPa/cWM7XPPbYYxo8ePBZ+zRv3lwxMTE6ePCgS/vJkyeVn5+vmJiYSl8XExOjkpISFRQUuPyWmpeX53xNgwYNJElt2rRxbo+KilK9evW0Z8+e8zgj76nusVqxYoU2bdqk+fPnS/pt5YIk1atXT3/96181ceLE8zwzz6vusSq3detW9e7dW8OHD9dTTz11XudSHerVq6eAgIAKq50qO8dyMTExZ+1f/r95eXnO/+7Kn3fs2NGD1dvnjfEqVx48fvzxR61YscKnr3pI3hmrL7/8UgcPHnS5Invq1Ck99thjmj59unbv3u3Zk/B11T3pxJeUTwxcv369s23p0qVVmhg4f/58Z9u2bdtcJgbm5OQYSS4TTg8fPmz8/f3N0qVLvXQ23uWtsdqxY4fZtGmT8/Haa68ZSebrr78+4yz1i523xsoYYzZv3mzq169vxowZ470T8KLOnTubhx56yPn81KlTpmHDhmedFNi3b1+XtoSEhAoTTqdOnercXlhYWKMmnHpyvIwxpqSkxPTv39+0bdvWHDx40DuFVwNPj9WhQ4dc/m3atGmTiY2NNU8++aTZtm2b907ERxE+3HTzzTebq666yqxdu9Z89dVXpmXLli5LIvft22euuOIKs3btWmfbiBEjTOPGjc2KFSvM+vXrTUJCgklISHDZb79+/Uzbtm3N6tWrzaZNm0zfvn1NmzZtTElJibVz8zRvjdXpVq5c6fOrXYzxzlht2rTJREVFmUGDBpkDBw44H770ATJv3jzjcDjMnDlzzNatW83w4cNNRESEyc3NNcYYc99995mxY8c6+69evdoEBgaaqVOnmu+//95MmDCh0qW2ERERZtGiReY///mP6devX41aauvJ8SopKTG33XabadSokdm4caPL+6i4uLhaztFTvPHe+j1Wu5wZ4cNNhw8fNgMHDjShoaEmLCzMDBkyxBw5csS5fdeuXUaSWblypbPt+PHjZuTIkebSSy81l1xyibn99tvNgQMHXPZbWFhohg4daiIiIkxkZKS5/fbbzZ49e2ydlld4a6xOV1PChzfGasKECUZShUeTJk0sntmFe+mll0zjxo1NUFCQ6dy5s1mzZo1zW48ePUxaWppL//fee89cfvnlJigoyLRt29Z89NFHLtvLysrM008/baKjo43D4TC9e/c2OTk5Nk7FCk+OV/n7rrLH6e9FX+Xp99bvET7OzM+Y/+9LcwAAAAtY7QIAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMCq/wcYs7FMDeUgQAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAGzCAYAAADjbSfcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyyklEQVR4nO3dfXzN9f/H8eeZ2YY5m8uN2EbkKoQyI/HVCl+VshJfFX5KF4uvpLJf5ap+bdE3ym0ot9KFn4RvlK+rSugbI6RvLiJEFJtS21xt2N6/P9x2fh3b2NnO2by3x/12Ozc778/7vM/rfc4unt7n8z7HYYwxAgAAuML5lXUBAAAARUFoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBUCQHDx6Uw+HQO++8U6T+DodDEyZM8Ph+1q5dK4fDobVr17rahgwZoqioKI/HKglP5wvA9wgtKPf279+vhx9+WI0bN1ZQUJCcTqe6dOmi1157TWfOnPHJfc6bN0/Tpk3zydglMWHCBDkcDv32229eGW/58uXFCiblxciRI+VwOLRv375C+zz77LNyOBz67rvvvHrfM2bMIFChwiG0oFxbtmyZWrdurQULFuj222/X9OnTlZiYqIiICD311FP6+9//7pP7vVJDS0lERkbqzJkzuv/++11ty5cv18SJEwvsf+bMGT333HNeue/Zs2drz549XhnLmwYNGiTpwvNdmA8++ECtW7dWmzZtvHrfhBZURP5lXQDgKwcOHNCAAQMUGRmpL774QvXq1XMdi4+P1759+7Rs2bIyrNAuDodDQUFBRe7vSd/LqVy5stfG8qbo6Gg1adJEH3zwgcaNG5fveEpKig4cOKCkpKQyqM5zWVlZCggIkJ8f/5/FlYnvTJRbkydP1smTJ/XWW2+5BZY8TZo0ca20XOr8hYvPzThx4oRGjRqlqKgoBQYGqm7durrlllv0zTffSJK6d++uZcuW6aeffpLD4ZDD4XA7H+PYsWMaNmyYwsLCFBQUpLZt2+rdd991u8+8el555RUlJyercePGqlq1qm699VYdPnxYxhi98MILatCggapUqaK+ffvq999/L9bj1L17d1177bXatWuX/vKXv6hq1aq66qqrNHny5AJrynuMhgwZouTkZNdjlHcp7HH76aef9Nhjj6lZs2aqUqWKatWqpXvuuUcHDx68bI0Xn9PSvXt3t/v88+XPz2F6erpGjRqlhg0bKjAwUE2aNNHLL7+s3Nxct/HT09M1ZMgQhYSEKDQ0VIMHD1Z6enqRHr9BgwZp9+7druf/z+bNmyeHw6GBAwdKkrKzszV+/Hg1adJEgYGBatiwoZ5++mllZ2fnu+3cuXPVsWNHVa1aVTVq1NBNN92kTz/9VJIUFRWlnTt3at26da55d+/e3XXbH3/8Uffcc49q1qypqlWrqlOnTvkCet65Q/Pnz9dzzz2nq666SlWrVlVmZmaR5g2UBVZaUG4tXbpUjRs3VufOnb067iOPPKJFixbp8ccfV8uWLXX8+HF99dVX+v7779W+fXs9++yzysjI0M8//6ypU6dKkoKDgyVdeMmke/fu2rdvnx5//HE1atRICxcu1JAhQ5Senp7v5ar//d//1dmzZzVixAj9/vvvmjx5svr3768ePXpo7dq1euaZZ7Rv3z5Nnz5dY8aM0dtvv12sOf3xxx/q1auX+vXrp/79+2vRokV65pln1Lp1a/Xu3bvA2zz88MM6cuSIPvvsM73//vuXvY/Nmzdrw4YNGjBggBo0aKCDBw9q5syZ6t69u3bt2qWqVasWud5nn31WDz74oFvb3LlztWrVKtWtW1eSdPr0aXXr1k2//PKLHn74YUVERGjDhg1KSEjQ0aNHXS/fGWPUt29fffXVV3rkkUfUokULLV68WIMHDy5SLYMGDdLEiRM1b948tW/f3tWek5OjBQsWqGvXroqIiFBubq7uuOMOffXVVxo+fLhatGih7du3a+rUqfrhhx+0ZMkS120nTpyoCRMmqHPnzpo0aZICAgK0adMmffHFF7r11ls1bdo0jRgxQsHBwXr22WclSWFhYZKktLQ0de7cWadPn9bIkSNVq1Ytvfvuu7rjjju0aNEi3XXXXW71v/DCCwoICNCYMWOUnZ2tgICAIj8PQKkzQDmUkZFhJJm+ffsWqf+BAweMJDNnzpx8xySZ8ePHu66HhISY+Pj4S47Xp08fExkZma992rRpRpKZO3euq+3s2bMmJibGBAcHm8zMTLd66tSpY9LT0119ExISjCTTtm1bc+7cOVf7wIEDTUBAgMnKyrpkXePHjzeSzK+//upq69atm5Fk3nvvPVdbdna2CQ8PN3Fxca62gh6j+Ph4U9ivkYsft9OnT+frk5KSku++16xZYySZNWvWuNoGDx5c4OOZZ/369aZy5crmv/7rv1xtL7zwgqlWrZr54Ycf3PqOHTvWVKpUyRw6dMgYY8ySJUuMJDN58mRXn/Pnz5uuXbsW+j1xsRtuuME0aNDA5OTkuNpWrlxpJJk33njDGGPM+++/b/z8/My///1vt9vOmjXLSDLr1683xhizd+9e4+fnZ+666y638YwxJjc31/V1q1atTLdu3fLVMmrUKCPJ7X5OnDhhGjVqZKKiolxj5j3OjRs3LvC5Aa5EvDyEcilvibt69epeHzs0NFSbNm3SkSNHPL7t8uXLFR4e7nq5QLpwvsbIkSN18uRJrVu3zq3/Pffco5CQENf16OhoSdJ9990nf39/t/azZ8/ql19+8bgm6cJK0H333ee6HhAQoI4dO+rHH38s1ngFqVKliuvrc+fO6fjx42rSpIlCQ0MLfGmlqFJTU3X33Xfruuuu04wZM1ztCxcuVNeuXVWjRg399ttvrktsbKxycnL05ZdfSrrwnPj7++vRRx913bZSpUoaMWJEkWu477779PPPP7vGlC68NBQQEKB77rnHVU+LFi3UvHlzt3p69OghSVqzZo0kacmSJcrNzdW4cePynVvy55ffCrN8+XJ17NhRN954o6stODhYw4cP18GDB7Vr1y63/oMHD3Z7boArGaEF5ZLT6ZR04fwTb5s8ebJ27Nihhg0bqmPHjpowYUKR/7j/9NNPatq0ab4/Ri1atHAd/7OIiAi363kBpmHDhgW2//HHH0WfyJ80aNAg3x/EGjVqFHu8gpw5c0bjxo1znV9Su3Zt1alTR+np6crIyCjWmOfPn1f//v2Vk5Ojjz76SIGBga5je/fu1cqVK1WnTh23S2xsrKQL5xZJFx7zevXquV7Cy9OsWbMi1zFgwABVqlTJtYsoKytLixcvVu/evVWjRg1XPTt37sxXzzXXXONWz/79++Xn56eWLVsW6zH56aefCqy9sO+xRo0aFet+gLLAOS0ol5xOp+rXr68dO3YUqX9h/4PNycnJ19a/f3917dpVixcv1qeffqopU6bo5Zdf1kcffVTo+R/FValSJY/ajTFevZ/ijleQESNGaM6cORo1apRiYmIUEhIih8OhAQMG5DsxtqieeuoppaSk6PPPP1eDBg3cjuXm5uqWW27R008/XeBt88KCN+SdjP3Pf/5TycnJWrp0qU6cOOHaEp1XT+vWrfXqq68WOMbFQbS0sMoCmxBaUG7ddtttevPNN5WSkqKYmJhL9s373/DFO0Yu/l9pnnr16umxxx7TY489pmPHjql9+/b6n//5H1doKSwERUZG6rvvvlNubq7basvu3btdx21SlJcr8ixatEiDBw/WP/7xD1dbVlZWkXfpXGz+/PmaNm2apk2bpm7duuU7fvXVV+vkyZOulZXCREZGavXq1Tp58qTbaoun7wszaNAgrVy5UitWrNC8efPkdDp1++23u9Xzn//8RzfffPMlH7err75aubm52rVrl6677rpC+13qe6yg2m39HgP+jJeHUG49/fTTqlatmh588EGlpaXlO75//3699tprki6szNSuXdvtnARJbudISBdWXi5+KaNu3bqqX7++27bVatWqFfiSx1//+lelpqbqww8/dLWdP39e06dPV3BwcIF/fK9k1apVk5Q/7BWkUqVK+VZupk+fXuBq1uXs2LFDDz74oO67775C3yCwf//+SklJ0apVq/IdS09P1/nz5yVdeE7Onz+vmTNnuo7n5ORo+vTpHtV05513qmrVqpoxY4ZWrFihfv36ub1XTf/+/fXLL79o9uzZ+W575swZnTp1yjWOn5+fJk2alG8F6s+PX7Vq1Qp83P/617/q66+/VkpKiqvt1KlTevPNNxUVFVXsl52AKwErLSi3rr76as2bN0/33nuvWrRooQceeEDXXnutzp49qw0bNri2Gud58MEHlZSUpAcffFDXX3+9vvzyS/3www9uY544cUINGjTQ3XffrbZt2yo4OFiff/65Nm/e7LaC0KFDB3344YcaPXq0brjhBgUHB+v222/X8OHD9cYbb2jIkCHaunWroqKitGjRIq1fv17Tpk3zyYnDvtShQwdJF97OvmfPnqpUqZIGDBhQYN/bbrtN77//vkJCQtSyZUvXyzq1atXy+H6HDh0qSbrppps0d+5ct2OdO3dW48aN9dRTT+mTTz7RbbfdpiFDhqhDhw46deqUtm/frkWLFungwYOqXbu2br/9dnXp0kVjx47VwYMH1bJlS3300Ucen2cTHBysO++803Vey59fGpKk+++/XwsWLNAjjzyiNWvWqEuXLsrJydHu3bu1YMECrVq1Stdff72aNGmiZ599Vi+88IK6du2qfv36KTAwUJs3b1b9+vWVmJgo6cJjP3PmTL344otq0qSJ6tatqx49emjs2LH64IMP1Lt3b40cOVI1a9bUu+++qwMHDuif//wnbxwHu5Xt5iXA93744Qfz0EMPmaioKBMQEGCqV69uunTpYqZPn+62Rfj06dNm2LBhJiQkxFSvXt3079/fHDt2zG3rbnZ2tnnqqadM27ZtTfXq1U21atVM27ZtzYwZM9zu8+TJk+Zvf/ubCQ0NNZLctuumpaWZoUOHmtq1a5uAgADTunXrfNtq87YXT5kyxa09b5vqwoUL3drnzJljJJnNmzdf8rEobMtzq1at8vW9eJtxQVuez58/b0aMGGHq1KljHA6H2/ZnXbTl+Y8//nDNOzg42PTs2dPs3r3bREZGmsGDB+eb46W2PEdGRhpJBV7+XN+JEydMQkKCadKkiQkICDC1a9c2nTt3Nq+88oo5e/asq9/x48fN/fffb5xOpwkJCTH333+/2bZtW5G3POdZtmyZkWTq1auXb7uyMRe2t7/88sumVatWJjAw0NSoUcN06NDBTJw40WRkZLj1ffvtt027du1c/bp162Y+++wz1/HU1FTTp08fU716dSPJbfvz/v37zd13321CQ0NNUFCQ6dixo/nXv/7lNn5h30vAlcxhjBfPtAMAAPAR1gkBAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxwxb25XG5uro4cOaLq1at79BbhAACg7BhjdOLECdWvX99nb2J4xYWWI0eOlNkHhwEAgJI5fPhwvg8w9ZYrLrTkvY354cOH5XQ6y7gaAABQFJmZmWrYsKFPP47kigsteS8JOZ1OQgsAAJbx5akdnIgLAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAX/si4AAC4WNXbZZfscTOpTCpUAuJKw0gIAAKzgcWj55ZdfdN9996lWrVqqUqWKWrdurS1btriOG2M0btw41atXT1WqVFFsbKz27t3r1aIBAEDF41Fo+eOPP9SlSxdVrlxZK1as0K5du/SPf/xDNWrUcPWZPHmyXn/9dc2aNUubNm1StWrV1LNnT2VlZXm9eAAAUHF4dE7Lyy+/rIYNG2rOnDmutkaNGrm+NsZo2rRpeu6559S3b19J0nvvvaewsDAtWbJEAwYM8FLZAACgovFopeWTTz7R9ddfr3vuuUd169ZVu3btNHv2bNfxAwcOKDU1VbGxsa62kJAQRUdHKyUlpcAxs7OzlZmZ6XYBAAC4mEeh5ccff9TMmTPVtGlTrVq1So8++qhGjhypd999V5KUmpoqSQoLC3O7XVhYmOvYxRITExUSEuK6NGzYsDjzAAAA5ZxHoSU3N1ft27fXSy+9pHbt2mn48OF66KGHNGvWrGIXkJCQoIyMDNfl8OHDxR4LAACUXx6Flnr16qlly5ZubS1atNChQ4ckSeHh4ZKktLQ0tz5paWmuYxcLDAyU0+l0uwAAAFzMo9DSpUsX7dmzx63thx9+UGRkpKQLJ+WGh4dr9erVruOZmZnatGmTYmJivFAuAACoqDzaPfTEE0+oc+fOeumll9S/f399/fXXevPNN/Xmm29KkhwOh0aNGqUXX3xRTZs2VaNGjfT888+rfv36uvPOO31RPwAAqCA8Ci033HCDFi9erISEBE2aNEmNGjXStGnTNGjQIFefp59+WqdOndLw4cOVnp6uG2+8UStXrlRQUJDXiwcAABWHwxhjyrqIP8vMzFRISIgyMjI4vwWooErzs4f4nCPAO0rj7zefPQQAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBX8y7oAACgPosYuu2yfg0l9SqESoPxipQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACs4FFomTBhghwOh9ulefPmruNZWVmKj49XrVq1FBwcrLi4OKWlpXm9aAAAUPF4vNLSqlUrHT161HX56quvXMeeeOIJLV26VAsXLtS6det05MgR9evXz6sFAwCAisnf4xv4+ys8PDxfe0ZGht566y3NmzdPPXr0kCTNmTNHLVq00MaNG9WpU6eSVwsAACosj1da9u7dq/r166tx48YaNGiQDh06JEnaunWrzp07p9jYWFff5s2bKyIiQikpKYWOl52drczMTLcLAADAxTxaaYmOjtY777yjZs2a6ejRo5o4caK6du2qHTt2KDU1VQEBAQoNDXW7TVhYmFJTUwsdMzExURMnTixW8QCuLFFjl122z8GkPqVQCYDyyKPQ0rt3b9fXbdq0UXR0tCIjI7VgwQJVqVKlWAUkJCRo9OjRruuZmZlq2LBhscYCAADlV4m2PIeGhuqaa67Rvn37FB4errNnzyo9Pd2tT1paWoHnwOQJDAyU0+l0uwAAAFysRKHl5MmT2r9/v+rVq6cOHTqocuXKWr16tev4nj17dOjQIcXExJS4UAAAULF59PLQmDFjdPvttysyMlJHjhzR+PHjValSJQ0cOFAhISEaNmyYRo8erZo1a8rpdGrEiBGKiYlh5xAAACgxj0LLzz//rIEDB+r48eOqU6eObrzxRm3cuFF16tSRJE2dOlV+fn6Ki4tTdna2evbsqRkzZvikcAAAULF4FFrmz59/yeNBQUFKTk5WcnJyiYoCgCtJUXZFAfA9PnsIAABYgdACAACsQGgBAABWILQAAAArePyBiQBwJeAjA4CKh5UWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiBzx4CUG4V5fOJANiDlRYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACv5lXQAA4P9FjV1WpH4Hk/r4uBLgysNKCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAK7B7CLBYUXaasMsEQHnBSgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACuwewhAqSrqZ+uURxV57oA3sNICAACsUKLQkpSUJIfDoVGjRrnasrKyFB8fr1q1aik4OFhxcXFKS0sraZ0AAKCCK3Zo2bx5s9544w21adPGrf2JJ57Q0qVLtXDhQq1bt05HjhxRv379SlwoAACo2IoVWk6ePKlBgwZp9uzZqlGjhqs9IyNDb731ll599VX16NFDHTp00Jw5c7RhwwZt3LixwLGys7OVmZnpdgEAALhYsU7EjY+PV58+fRQbG6sXX3zR1b5161adO3dOsbGxrrbmzZsrIiJCKSkp6tSpU76xEhMTNXHixOKUAcBL+DgAADbweKVl/vz5+uabb5SYmJjvWGpqqgICAhQaGurWHhYWptTU1ALHS0hIUEZGhuty+PBhT0sCAAAVgEcrLYcPH9bf//53ffbZZwoKCvJKAYGBgQoMDPTKWAAAoPzyaKVl69atOnbsmNq3by9/f3/5+/tr3bp1ev311+Xv76+wsDCdPXtW6enpbrdLS0tTeHi4N+sGAAAVjEcrLTfffLO2b9/u1jZ06FA1b95czzzzjBo2bKjKlStr9erViouLkyTt2bNHhw4dUkxMjPeqBgAAFY5HoaV69eq69tpr3dqqVaumWrVqudqHDRum0aNHq2bNmnI6nRoxYoRiYmIKPAkXAACgqLz+Nv5Tp06Vn5+f4uLilJ2drZ49e2rGjBnevhsARcRbxwMoL0ocWtauXet2PSgoSMnJyUpOTi7p0AAAAC589hAAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBf+yLgCAHaLGLivrEgBUcKy0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFbwL+sCAACeixq77LJ9Dib1KYVKgNLDSgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAVvAotMycOVNt2rSR0+mU0+lUTEyMVqxY4TqelZWl+Ph41apVS8HBwYqLi1NaWprXiwYAABWPR6GlQYMGSkpK0tatW7Vlyxb16NFDffv21c6dOyVJTzzxhJYuXaqFCxdq3bp1OnLkiPr16+eTwgEAQMXiMMaYkgxQs2ZNTZkyRXfffbfq1KmjefPm6e6775Yk7d69Wy1atFBKSoo6depUpPEyMzMVEhKijIwMOZ3OkpQGlHtRY5eVdQm4gh1M6lPWJaACKY2/38U+pyUnJ0fz58/XqVOnFBMTo61bt+rcuXOKjY119WnevLkiIiKUkpJS6DjZ2dnKzMx0uwAAAFzM39MbbN++XTExMcrKylJwcLAWL16sli1b6ttvv1VAQIBCQ0Pd+oeFhSk1NbXQ8RITEzVx4kSPCwfKO1ZRAMCdxystzZo107fffqtNmzbp0Ucf1eDBg7Vr165iF5CQkKCMjAzX5fDhw8UeCwAAlF8er7QEBASoSZMmkqQOHTpo8+bNeu2113Tvvffq7NmzSk9Pd1ttSUtLU3h4eKHjBQYGKjAw0PPKAQBAhVLi92nJzc1Vdna2OnTooMqVK2v16tWuY3v27NGhQ4cUExNT0rsBAAAVnEcrLQkJCerdu7ciIiJ04sQJzZs3T2vXrtWqVasUEhKiYcOGafTo0apZs6acTqdGjBihmJiYIu8cAgAAKIxHoeXYsWN64IEHdPToUYWEhKhNmzZatWqVbrnlFknS1KlT5efnp7i4OGVnZ6tnz56aMWOGTwoHAAAVS4nfp8XbeJ8W2K4ou36K8v4Z7B5CaeC9XOAtV/T7tAAAAJQmQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACv5lXQAAoOxEjV122T4Hk/qUQiXA5bHSAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAX/si4AsEnU2GVX1DgAUJGw0gIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsIJHoSUxMVE33HCDqlevrrp16+rOO+/Unj173PpkZWUpPj5etWrVUnBwsOLi4pSWlubVogEAQMXjUWhZt26d4uPjtXHjRn322Wc6d+6cbr31Vp06dcrV54knntDSpUu1cOFCrVu3TkeOHFG/fv28XjgAAKhYPHpH3JUrV7pdf+edd1S3bl1t3bpVN910kzIyMvTWW29p3rx56tGjhyRpzpw5atGihTZu3KhOnTp5r3IAAFChlOicloyMDElSzZo1JUlbt27VuXPnFBsb6+rTvHlzRUREKCUlpcAxsrOzlZmZ6XYBAAC4WLE/eyg3N1ejRo1Sly5ddO2110qSUlNTFRAQoNDQULe+YWFhSk1NLXCcxMRETZw4sbhlAACuAEX5PK2DSX1KoRKUZ8VeaYmPj9eOHTs0f/78EhWQkJCgjIwM1+Xw4cMlGg8AAJRPxVppefzxx/Wvf/1LX375pRo0aOBqDw8P19mzZ5Wenu622pKWlqbw8PACxwoMDFRgYGBxygAAABWIRystxhg9/vjjWrx4sb744gs1atTI7XiHDh1UuXJlrV692tW2Z88eHTp0SDExMd6pGAAAVEgerbTEx8dr3rx5+vjjj1W9enXXeSohISGqUqWKQkJCNGzYMI0ePVo1a9aU0+nUiBEjFBMTw84hAABQIh6FlpkzZ0qSunfv7tY+Z84cDRkyRJI0depU+fn5KS4uTtnZ2erZs6dmzJjhlWIBAEDF5VFoMcZctk9QUJCSk5OVnJxc7KKAslCU3Q8AgLLDZw8BAAArEFoAAIAVCC0AAMAKhBYAAGCFYr+NPwCgYuAkdVwpWGkBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAK/mVdAFAaosYuK+sSAAAlxEoLAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFYgtAAAACt4HFq+/PJL3X777apfv74cDoeWLFnidtwYo3HjxqlevXqqUqWKYmNjtXfvXm/VCwAAKiiPQ8upU6fUtm1bJScnF3h88uTJev311zVr1ixt2rRJ1apVU8+ePZWVlVXiYgEAQMXl7+kNevfurd69exd4zBijadOm6bnnnlPfvn0lSe+9957CwsK0ZMkSDRgwoGTVAgCACsur57QcOHBAqampio2NdbWFhIQoOjpaKSkpBd4mOztbmZmZbhcAAICLebzScimpqamSpLCwMLf2sLAw17GLJSYmauLEid4sAyUQNXbZZfscTOpTCpVcUJR6AFQsV9rvKZSeMt89lJCQoIyMDNfl8OHDZV0SAAC4Ank1tISHh0uS0tLS3NrT0tJcxy4WGBgop9PpdgEAALiYV0NLo0aNFB4ertWrV7vaMjMztWnTJsXExHjzrgAAQAXj8TktJ0+e1L59+1zXDxw4oG+//VY1a9ZURESERo0apRdffFFNmzZVo0aN9Pzzz6t+/fq68847vVk3AACoYDwOLVu2bNFf/vIX1/XRo0dLkgYPHqx33nlHTz/9tE6dOqXhw4crPT1dN954o1auXKmgoCDvVQ0AsA4n0KKkPA4t3bt3lzGm0OMOh0OTJk3SpEmTSlQYAADAn5X57iEAAICiILQAAAArEFoAAIAVCC0AAMAKXn0bfwAAbMFuJvuw0gIAAKxAaAEAAFYgtAAAACsQWgAAgBUILQAAwArsHoJPcFY+gOIoyu8OG/E70TtYaQEAAFYgtAAAACsQWgAAgBUILQAAwAqEFgAAYAV2D8Fj3jq7v7zuEgAA+AYrLQAAwAqEFgAAYAVCCwAAsAKhBQAAWIETcQEA5U5pbhjg7fdLDystAADACoQWAABgBUILAACwAqEFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACnz1UTvD5GABgN36PXx4rLQAAwAqEFgAAYAVCCwAAsAKhBQAAWIHQAgAArMDuoTLG2eIAYLei/B6Hd7DSAgAArEBoAQAAViC0AAAAKxBaAACAFQgtAADAChVu95CNu3W8dWY6Z7gDQPln49+5ovLZSktycrKioqIUFBSk6Ohoff311766KwAAUAH4JLR8+OGHGj16tMaPH69vvvlGbdu2Vc+ePXXs2DFf3B0AAKgAfBJaXn31VT300EMaOnSoWrZsqVmzZqlq1ap6++23fXF3AACgAvD6OS1nz57V1q1blZCQ4Grz8/NTbGysUlJS8vXPzs5Wdna263pGRoYkKTMz09ulSZJys09fto+v7rsgRakHAACpaH+fyurvXN6Yxhivj53H66Hlt99+U05OjsLCwtzaw8LCtHv37nz9ExMTNXHixHztDRs29HZpRRYyrczuGgCAQnnr75Mv/86dOHFCISEhPhm7zHcPJSQkaPTo0a7rubm5+v3331WrVi05HI4yrKxwmZmZatiwoQ4fPiyn01nW5fgEcywfyvscy/v8JOZYHpT3+Un/P8ddu3apfv36Prsfr4eW2rVrq1KlSkpLS3NrT0tLU3h4eL7+gYGBCgwMdGsLDQ31dlk+4XQ6y+03YB7mWD6U9zmW9/lJzLE8KO/zk6SrrrpKfn6+ews4r48cEBCgDh06aPXq1a623NxcrV69WjExMd6+OwAAUEH45OWh0aNHa/Dgwbr++uvVsWNHTZs2TadOndLQoUN9cXcAAKAC8Elouffee/Xrr79q3LhxSk1N1XXXXaeVK1fmOznXVoGBgRo/fny+l7XKE+ZYPpT3OZb3+UnMsTwo7/OTSm+ODuPLvUkAAABewgcmAgAAKxBaAACAFQgtAADACoQWAABgBUILAACwAqGlEL///rsGDRokp9Op0NBQDRs2TCdPnrxk/xEjRqhZs2aqUqWKIiIiNHLkSNcHQOY5dOiQ+vTpo6pVq6pu3bp66qmndP78eV9Pp8B6PZmfJL355pvq3r27nE6nHA6H0tPT8/WJioqSw+FwuyQlJfloFpfmqzkWZ1xfKU4tWVlZio+PV61atRQcHKy4uLh872B98XPocDg0f/58X07FJTk5WVFRUQoKClJ0dLS+/vrrS/ZfuHChmjdvrqCgILVu3VrLly93O26M0bhx41SvXj1VqVJFsbGx2rt3ry+ncFnenuOQIUPyPV+9evXy5RQuyZP57dy5U3Fxca7fHdOmTSvxmKXB23OcMGFCvuewefPmPpzB5Xkyx9mzZ6tr166qUaOGatSoodjY2Hz9vfKzaFCgXr16mbZt25qNGzeaf//736ZJkyZm4MCBhfbfvn276devn/nkk0/Mvn37zOrVq03Tpk1NXFycq8/58+fNtddea2JjY822bdvM8uXLTe3atU1CQkJpTMmNp/MzxpipU6eaxMREk5iYaCSZP/74I1+fyMhIM2nSJHP06FHX5eTJkz6axaX5ao7FGddXilPLI488Yho2bGhWr15ttmzZYjp16mQ6d+7s1keSmTNnjtvzeObMGV9OxRhjzPz5801AQIB5++23zc6dO81DDz1kQkNDTVpaWoH9169fbypVqmQmT55sdu3aZZ577jlTuXJls337dlefpKQkExISYpYsWWL+85//mDvuuMM0atSoVOZTEF/McfDgwaZXr15uz9fvv/9eWlNy4+n8vv76azNmzBjzwQcfmPDwcDN16tQSj+lrvpjj+PHjTatWrdyew19//dXHMymcp3P829/+ZpKTk822bdvM999/b4YMGWJCQkLMzz//7OrjjZ9FQksBdu3aZSSZzZs3u9pWrFhhHA6H+eWXX4o8zoIFC0xAQIA5d+6cMcaY5cuXGz8/P5OamurqM3PmTON0Ok12drb3JnAZJZ3fmjVrLhlaCvqBLG2+mqO3vje8oTi1pKenm8qVK5uFCxe62r7//nsjyaSkpLjaJJnFixf7rPbCdOzY0cTHx7uu5+TkmPr165vExMQC+/fv39/06dPHrS06Oto8/PDDxhhjcnNzTXh4uJkyZYrreHp6ugkMDDQffPCBD2Zwed6eozEXQkvfvn19Uq+nPJ3fnxX2+6MkY/qCL+Y4fvx407ZtWy9WWTIlfczPnz9vqlevbt59911jjPd+Fnl5qAApKSkKDQ3V9ddf72qLjY2Vn5+fNm3aVORxMjIy5HQ65e/v7xq3devWbu8M3LNnT2VmZmrnzp3em8BleGt+hUlKSlKtWrXUrl07TZkypUxe/vLVHH392Pm6lq1bt+rcuXOKjY11tTVv3lwRERFKSUlx6xsfH6/atWurY8eOevvtt2V8/D6UZ8+e1datW91q8/PzU2xsbL7a8qSkpLj1ly78TOX1P3DggFJTU936hISEKDo6utAxfckXc8yzdu1a1a1bV82aNdOjjz6q48ePe38Cl1Gc+ZXFmCXhy3r27t2r+vXrq3Hjxho0aJAOHTpU0nKLxRtzPH36tM6dO6eaNWtK8t7Pok/ext92qampqlu3rlubv7+/atasqdTU1CKN8dtvv+mFF17Q8OHD3ca9+KMM8q4XdVxv8Mb8CjNy5Ei1b99eNWvW1IYNG5SQkKCjR4/q1VdfLdG4nvLVHH352JVGLampqQoICMj3SephYWFut5k0aZJ69OihqlWr6tNPP9Vjjz2mkydPauTIkV6fR57ffvtNOTk5Bf6M7N69u8DbFPYzlTeXvH8v1ac0+WKOktSrVy/169dPjRo10v79+/Xf//3f6t27t1JSUlSpUiXvT6QQxZlfWYxZEr6qJzo6Wu+8846aNWumo0ePauLEieratat27Nih6tWrl7Rsj3hjjs8884zq16/vCine+lmsUKFl7Nixevnlly/Z5/vvvy/x/WRmZqpPnz5q2bKlJkyYUOLxiqq05ncpo0ePdn3dpk0bBQQE6OGHH1ZiYqJXPpPiSpijr10Jc3z++eddX7dr106nTp3SlClTfBpaUHwDBgxwfd26dWu1adNGV199tdauXaubb765DCtDUfXu3dv1dZs2bRQdHa3IyEgtWLBAw4YNK8PKPJeUlKT58+dr7dq1CgoK8urYFSq0PPnkkxoyZMgl+zRu3Fjh4eE6duyYW/v58+f1+++/Kzw8/JK3P3HihHr16qXq1atr8eLFqly5sutYeHh4vrOp83ZtXG7coiiN+XkqOjpa58+f18GDB9WsWbMSj1fWcyyNx86XcwwPD9fZs2eVnp7uttqSlpZ2yfqjo6P1wgsvKDs722cfiFa7dm1VqlQp306mS9UWHh5+yf55/6alpalevXpufa677jovVl80vphjQRo3bqzatWtr3759pRpaijO/shizJEqrntDQUF1zzTXat2+f18YsqpLM8ZVXXlFSUpI+//xztWnTxtXutZ/FIp/9UoHkneC4ZcsWV9uqVasue7JlRkaG6dSpk+nWrZs5depUvuN5J+L++ezrN954wzidTpOVleXdSVxCceeX51In4l5s7ty5xs/Pr9R3MvhqjiUd15uKU0veibiLFi1yte3evTvfibgXe/HFF02NGjW8V3whOnbsaB5//HHX9ZycHHPVVVdd8iTV2267za0tJiYm34m4r7zyiut4RkZGmZ+I6805FuTw4cPG4XCYjz/+2DtFe8DT+f3ZpU7ELe6YvuCLOV7sxIkTpkaNGua1114rSanFVpw5vvzyy8bpdBb4u8RbP4uElkL06tXLtGvXzmzatMl89dVXpmnTpm5bSX/++WfTrFkzs2nTJmPMhQc/OjratG7d2uzbt89t29r58+eNMf+/5fnWW2813377rVm5cqWpU6dOmW159mR+xhhz9OhRs23bNjN79mwjyXz55Zdm27Zt5vjx48YYYzZs2GCmTp1qvv32W7N//34zd+5cU6dOHfPAAw+U+vyM8c0cizJuaSrOHB955BETERFhvvjiC7NlyxYTExNjYmJiXMc/+eQTM3v2bLN9+3azd+9eM2PGDFO1alUzbtw4n89n/vz5JjAw0Lzzzjtm165dZvjw4SY0NNS14+7+++83Y8eOdfVfv3698ff3N6+88or5/vvvzfjx4wvc8hwaGmo+/vhj891335m+ffuW+ZZnb87xxIkTZsyYMSYlJcUcOHDAfP7556Z9+/amadOmpfqfoeLOLzs722zbts1s27bN1KtXz4wZM8Zs27bN7N27t8hjljZfzPHJJ580a9euNQcOHDDr1683sbGxpnbt2ubYsWOlPj9jPJ9jUlKSCQgIMIsWLXL7+3fixAm3PiX9WSS0FOL48eNm4MCBJjg42DidTjN06FC3B//AgQNGklmzZo0x5v//Z17Q5cCBA67bHTx40PTu3dtUqVLF1K5d2zz55JOuLdGlydP5GXNhS15B85szZ44xxpitW7ea6OhoExISYoKCgkyLFi3MSy+9VCa/OI3xzRyLMm5pKs4cz5w5Yx577DFTo0YNU7VqVXPXXXeZo0ePuo6vWLHCXHfddSY4ONhUq1bNtG3b1syaNcvk5OSUypymT59uIiIiTEBAgOnYsaPZuHGj61i3bt3M4MGD3fovWLDAXHPNNSYgIMC0atXKLFu2zO14bm6uef75501YWJgJDAw0N998s9mzZ09pTKVQ3pzj6dOnza233mrq1KljKleubCIjI81DDz1UZn/QjfFsfnnfoxdfunXrVuQxy4K353jvvfeaevXqmYCAAHPVVVeZe++91+zbt68UZ5SfJ3OMjIwscI7jx4939fHGz6LDGB/vYwQAAPAC3qcFAABYgdACAACsQGgBAABWILQAAAArEFoAAIAVCC0AAMAKhBYAAGAFQgsAALACoQUAAFiB0AIAAKxAaAEAAFb4P93azw7FSIeLAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# FOR CORONAVIRUS:\n",
    "maskedlm_model.resize_token_embeddings(len(tokenizer)) \n",
    "random_vector = maskedlm_model.get_input_embeddings().weight[-1].detach().numpy() # last one is covid random vector\n",
    "\n",
    "plt.title(\"Randomly Initialized Vector\")\n",
    "plt.hist(random_vector, bins=50)\n",
    "plt.show()\n",
    "\n",
    "pandemic_id = tokenizer.convert_tokens_to_ids(\"pandemic\")\n",
    "virus_id = tokenizer.convert_tokens_to_ids(\"virus\")\n",
    "respiratory_id = tokenizer.convert_tokens_to_ids(\"respiratory\")\n",
    "virus_embedding = maskedlm_model.get_input_embeddings().weight[virus_id]\n",
    "pandemic_embedding = maskedlm_model.get_input_embeddings().weight[pandemic_id]\n",
    "respiratory_embedding = maskedlm_model.get_input_embeddings().weight[respiratory_id]\n",
    "mean_embedding = torch.mean(torch.stack([virus_embedding, respiratory_embedding, pandemic_embedding]), dim=0)\n",
    "maskedlm_model.get_input_embeddings().weight[-1].data[:] = mean_embedding\n",
    "mean_vector = maskedlm_model.get_input_embeddings().weight[-1].detach().numpy()\n",
    "\n",
    "plt.title(\"Custom Initialized Vector\")\n",
    "plt.hist(mean_vector, bins=50)\n",
    "plt.show()\n",
    "\n",
    "# For COVID:\n",
    "maskedlm_model.resize_token_embeddings(len(tokenizer)) \n",
    "random_vector = maskedlm_model.get_input_embeddings().weight[-2].detach().numpy() # last one is covid random vector\n",
    "\n",
    "mean_embedding = torch.mean(torch.stack([virus_embedding, respiratory_embedding, pandemic_embedding]), dim=0)\n",
    "maskedlm_model.get_input_embeddings().weight[-2].data[:] = mean_embedding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = LineByLineTextDataset(\n",
    "    tokenizer=tokenizer,\n",
    "    file_path=\"../../data/pretraining_tweets_en_full_clean.txt\",\n",
    "    block_size=32,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom DataCollator for Language Modeling\n",
    "class CustomDataCollatorForLM(DataCollatorForLanguageModeling):\n",
    "\n",
    "    def __init__(self, tokenizer, mlm=True, mlm_probability=0.15):\n",
    "        super().__init__(tokenizer, mlm=mlm, mlm_probability=mlm_probability)\n",
    "        self.custom_masked_tokens = []\n",
    "        self.custom_masked_probability = mlm_probability\n",
    "\n",
    "    def add_custom_masked_tokens(self, tokens, probability):\n",
    "        self.custom_masked_tokens.extend(tokens)\n",
    "        self.custom_masked_probability = probability\n",
    "    \n",
    "    def collate_batch(self, features):\n",
    "        inputs = self.tokenizer.pad({\"input_ids\": [f.input_ids for f in features]}, return_tensors=\"pt\")\n",
    "        labels = inputs[\"input_ids\"].clone() \n",
    "\n",
    "        for i, feature in enumerate(features):\n",
    "            masked_indices = random.sample(range(len(feature.input_ids)), int(self.custom_masked_probability * len(feature.input_ids)))\n",
    "\n",
    "            for idx in masked_indices:\n",
    "                if feature.input_ids[idx] in self.custom_masked_tokens:\n",
    "                    labels[i, idx] = self.tokenizer.mask_token_id\n",
    "\n",
    "        return {\"input_ids\": inputs[\"input_ids\"], \"attention_mask\": inputs[\"attention_mask\"], \"labels\": labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up training arguments\n",
    "training_args = TrainingArguments(   \n",
    "    output_dir=\"./\",\n",
    "    overwrite_output_dir=True,\n",
    "    learning_rate=3e-05, \n",
    "    num_train_epochs=1,\n",
    "    per_device_train_batch_size=32,\n",
    "    save_steps=10000,\n",
    "    #save_total_limit=2,\n",
    ")\n",
    "\n",
    "# Set up the Trainer with custom data collator\n",
    "trainer = Trainer(\n",
    "    model=maskedlm_model,\n",
    "    args=training_args,\n",
    "    data_collator=CustomDataCollatorForLM(tokenizer=tokenizer, mlm=True, mlm_probability=0.15),\n",
    "    train_dataset=dataset,\n",
    ")\n",
    "\n",
    "# Add custom masked tokens with probability\n",
    "custom_masked_tokens = [\"covid\", \"coronavirus\", \"cdc\"]\n",
    "custom_masked_probability = 0.15  # Adjust as needed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.data_collator.add_custom_masked_tokens(custom_masked_tokens, custom_masked_probability)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start pretraining\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.save_model(\"covid_roberta_15_masked\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.save_pretrained(\"covid_roberta_15_masked\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
